<?php

namespace App\Console\Commands;

use Illuminate\Console\Command;
use DiDom\Document;
use DiDom\Query;
use App\Model\WebBaike;
use App\Model\WebQue;
use App\Model\WebAnw;
use App\Model\WebUsername;
use App\Services\OSS\OSS;
class BeginSpider extends Command
{
    /**
     * The name and signature of the console command.
     *
     * @var string
     */
    protected $signature = 'command:spider';

    /**
     * The console command description.
     *
     * @var string
     */
    protected $description = 'Command description';

    /**
     * Create a new command instance.
     *
     * @return void
     */
public function __construct()
    {
        parent::__construct();
    }

    /**
     * Execute the console command.
     *
     * @return mixed
     */
    public function handle()
    {
      //$this->getto8toask();
    }
   


  public function getto8toask(){
      $from = 'to8to';
      $begin = 20;
      $endnum = 7541039;
      $WebQue =WebQue::select('ask_id')->orderBy('id','desc')->where('from',$from)->first();
      if($WebQue){
          $begin = $WebQue->ask_id>=$begin?$WebQue->ask_id+1:$begin;         
      }  
      for($i =$begin;$i<$endnum;$i++){
          $url ="http://www.to8to.com/ask/k".$i.".html";
          $document = new Document($url, true);
          $base_data = $document->find('.global_pie');          
          if(isset($base_data[0])){
            $navarray =  $base_data[0]->find("li"); 
            $sign_id = time().$i;
            $nav = '';
            foreach ($navarray as $key => $value) {         
              $nav .= $value->text().' > ';
            }
            $ask_qustion = $document->find('.ask_qustion')[0];
            $ques = $ask_qustion->find('.title3')[0]->text(); 
            if(isset($ask_qustion->find('.ask_one_p')[0])){
              $ques_des =$ask_qustion->find('.ask_one_p')[0]->text(); 
            }else{
              $ques_des='';
            }

            $WebQue = new WebQue();
            $WebQue->ask_id = $i;
            $WebQue->sign_id = $sign_id;
            $WebQue->title = $ques;
            $WebQue->des= $ques_des;
            $WebQue->from= $from;
            $WebQue->url= $url;
            $WebQue->save();            
            $Que_id =  $WebQue->id;    
            $bestData =$document->find('.best_answer'); 
            if (isset($bestData[0])) {
              $best_answer =$bestData[0]->find('.ask_one_p')[0]->text();        
              $WebAnw = new WebAnw();
              $WebAnw->que_id = $Que_id;
              $WebAnw->sign_id = $sign_id;
              $WebAnw->isbest = 1;
              $WebAnw->content = $best_answer;           
              $WebAnw->save();
            }               
            $otherAnswer = $document->find('.ask_answer_li');   
              if(count($otherAnswer)){
              foreach ($otherAnswer as $key => $value) {
                  if($key>0){
                    $username[] = $value->find(".company_name")[0]->text();
                    $answer = $value->find("p")[0]->text();
                    $WebAnw = new WebAnw();
                    $WebAnw->que_id = $Que_id;
                    $WebAnw->sign_id = $sign_id;
                    $WebAnw->content = $answer;  
                    $WebAnw->save();
                  }
                }
            }

            }
        
        }
}
    private function deleteimg(){
      for ($i=0; $i <20000 ; $i++) {
        $WebBaike =WebBaike::select('id','content')->where('from','to8to')->where('delete_log',0)->orderBy('id','desc')->first();
        if($WebBaike){


            $match = "/beijing.aliyuncs.com\/(.*)jpg[\"]+/";
            preg_match_all($match, $WebBaike->content, $matches);
            if(count($matches[1])){

               foreach ($matches[1] as $key => $value) {
                $url = substr($value,0,-1).'.jpg';
                try{
                  $newcontent = preg_replace('/zfnb-mis-imgs/', 'zfnb-zhufaner-com',$WebBaike->content);
                  $WebBaike = WebBaike::find($WebBaike->id);
                  $WebBaike->content = $newcontent;
                  $WebBaike->delete_log = 1;
                  $WebBaike->save();

                  OSS::moveObject('zfnb-mis-imgs',$url,'zfnb-zhufaner-com',$url);
                  OSS::publicDeleteObject('zfnb-mis-imgs',$url);

                }catch(Exception $e) 
                { 
                  continue;
                }
              }
            }else{
              $newcontent = preg_replace('/zfnb-mis-imgs/', 'zfnb-zhufaner-com',$WebBaike->content);
              $WebBaike = WebBaike::find($WebBaike->id);
              $WebBaike->content = $newcontent;
              $WebBaike->delete_log = 1;
              $WebBaike->save();  
            }
        }

      }
    
    }
    private function to8to(){
      $from = 'to8to';
      $begin = 1;
      $endnum =28500;
      $WebBaike =WebBaike::select('bkid')->orderBy('id','desc')->where('from',$from)->first();
      if($WebBaike){
          $begin = $WebBaike->bkid>=$begin?$WebBaike->bkid+1:$begin;         
      }
      for($i =$begin;$i<$endnum;$i++){
        $url = "http://www.to8to.com/baike/$i";
        $document = new Document($url, true);
        $navarray = $document->find('.small-nav-wrap'); 
        if(isset($navarray[0])){
            $navarray = $navarray[0]->find(".nav-block");
            $nav= '';
            foreach ($navarray as $key => $item) {
              if($key>1){
                $nav.='>'.$item->text();
              }
            }
            $title = $document->find('.bk-detail-content-title')[0]->text();
            $abstract = $document->find('.bk-detail-content-intro-right-small')[0]->text();
            $content = $document->find('.bk-detail-content')[0]->html();
            $match ="/>.*\&gt;\&gt;/U";
            $content= preg_replace($match, '', $content);
            
            $match ="/<[\/]?[a|h1|h2|strong](.*)>/U";
            $content  = preg_replace($match, '', $content);
            $match ="/<p(.*)>/U";
            $content= preg_replace($match, '<p>', $content);
            $content= $this->downRemoutPic($content);
            $createWebBaike =new WebBaike();
            $createWebBaike->bkid =$i;
            $createWebBaike->title =$title;
            $createWebBaike->url =$url;
            $createWebBaike->abstract =$abstract ;
            $createWebBaike->content =$content;
            $createWebBaike->nav =$nav;
            $createWebBaike->from =$from;
            $createWebBaike->save();
        }
      }
    }
  private function gettobosu(){
      $from = 'tobosu';
      $begin = 12;
      $endnum = 4500;
      $WebBaike =WebBaike::select('bkid')->orderBy('id','desc')->where('from',$from)->first();
      if($WebBaike){
          $begin = $WebBaike->bkid>=$begin?$WebBaike->bkid+1:$begin;         
      }
      for($i =$begin;$i<$endnum;$i++){
          $url = "http://baike.tobosu.com/view/n$i.html/";
          $document = new Document($url, true);
          if(isset($document->find('.ency-detail')[0])){
            $nav = $document->find('.ed-cur-page')[0]->find(".fl"); 
            $title = $document->find(".edw-intro")[0]->find('h1')[0]->text();
            $abstract = $document->find(".edw-intro")[0]->find('.intro')[0]->text();
            $content = $this->downRemoutPic($document->find(".ed-dl")[0]->html());
            $navcontent = '';
            foreach ( $nav as  $item) {
              $navcontent .=$item->find("a")[0]->text().">";             
            }
            $match ="/<[\/]?[a|h1|h2|strong](.*)>/U";
            $content  = preg_replace($match, '', $content);
            $match ="/<p(.*)>/U";
            $content= preg_replace($match, '<p>', $content);           
              $createWebBaike =new WebBaike();
               $createWebBaike->bkid =$i;
               $createWebBaike->title =$title;
               $createWebBaike->url =$url;
               $createWebBaike->abstract =$abstract ;
               $createWebBaike->content =$content;
               $createWebBaike->nav =$navcontent;
               $createWebBaike->from =$from;
               $createWebBaike->save();
          }
 
      } 
    }
  private function getqijia(){
      $from = 'qijia';
      $begin = 1471;
      $endnum = 8500;
      $WebBaike =WebBaike::select('bkid')->orderBy('id','desc')->where('from',$from)->first();
      if($WebBaike){
          $begin = $WebBaike->bkid>=$begin?$WebBaike->bkid+1:$begin;         
        }
          for($i =$begin;$i<$endnum;$i++){
              $url = "http://www.jia.com/baike/bdetail-$i/";
              $document = new Document($url, true);
              $nav = $document->find('.bk-nav'); 
              if(isset($nav[0])){
               $nav= $nav[0]->text();
              $title = $document->find('h1')[0]->text(); 
              $contents = $document->find('.atical-floor'); 
              $abstract =  $contents[0]->find("i")[0]->text();
              $num = count($contents);          
              $content='';
              for ($j=1; $j <$num-1 ; $j++) { 
                  if(isset($contents[$j]->find(".floor-node")[0])){
                     $content .= $contents[$j]->find(".floor-node")[0]->html();
                     $content .= $contents[$j]->find(".txt")[0]->html();
                  }
                 
                  if(isset($contents[$j]->find("img")[0])){
                     $content .= $contents[$j]->find("img")[0]->html();
                  }
                 
              }
              $match ="/<[\/]?[a|h1|h2|strong](.*)>/U";
              $content  = preg_replace($match, '', $content);
              $match ="/<p(.*)>/U";
              $content= preg_replace($match, '<p>', $content);
              $createWebBaike =new WebBaike();
              $createWebBaike->bkid =$i;
              $createWebBaike->title =$title;
              $createWebBaike->url =$url;
              $createWebBaike->abstract =$abstract ;
              $createWebBaike->content =$this->downRemoutPic($content);
              $createWebBaike->nav =$nav;
              $createWebBaike->from =$from;
              $createWebBaike->save();
        
             } 
           }
        
    }
    private function downRemoutPic($content)
  {

  
    $path = public_path().'/case/';
    $localPath =public_path();
    preg_match_all('/src=\"(\S)+\"/', $content, $matchesArr);   
   
    foreach($matchesArr[0] as $url) {

      $url = substr($url, 5, -1);

      if(stripos($url, 'aliyuncs') === false) {
        if(strpos($url, 'http') === false) {
          //本地上传的
          // 上传到阿里云上, 删除本地文件
          $resPic = $this->toImageService($localPath . $url);
  
          @unlink($localPath . $url);
          //更新content里面的链接地址
          $content = str_replace($url, $resPic, $content); 
        } else {
          //远程上传的
          $tmpPic=@file_get_contents($url);
          if($tmpPic){           
            $tmpName = time() . rand(100,999) . rand(100,999);
            // 存入本地
            if(file_put_contents($path . $tmpName, $tmpPic)) {
              // 上传到阿里云上, 删除本地文件
              $resPic = $this->toImageService($path . $tmpName);
              @unlink($path . $tmpName);
              //更新content里面的链接地址
              $content = str_replace($url, $resPic, $content);
            } else {
              $content = str_replace($url, '', $content);
            }
          }
        }
      }
    }  

    return $content;
  }  
  // 将图片通过curl传到图片服务器 $imgPath 带扩展名
  private function toImageService( $imgPath )
  {       
          //img_water_mark($imgPath,public_path().'/images/common/0.jpg');

          $mother = time();
          $fucker = md5( $mother . '394d722ab71dc3212180f4eb73b83919');
          $ch = curl_init('http://images.zhufaner.com/upload?mother='.$mother.'&fucker='.$fucker);
          $data['file'] = new \CURLFile( $imgPath );
          curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, FALSE);
          curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, FALSE);
          curl_setopt($ch, CURLOPT_POST,1);
          curl_setopt($ch, CURLOPT_POSTFIELDS, $data);
          curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
          curl_setopt($ch, CURLOPT_TIMEOUT,5);
          $output = curl_exec($ch);
          if ( curl_errno($ch) == 28 || $output == 'error' ) {
                  $output = 0;
          }
          curl_close($ch);
          $url = 'https://zfnb-zhufaner-com.oss-cn-beijing.aliyuncs.com/';
          $suffix = '.jpg';
          $resPic = $url . $output . $suffix;
          return $resPic;
  } 
}
